library(rvest)
library(tictoc)

rvest_zinc_catalog <- function(x, catalog = 'mcee'){
  url <- str_glue('https://zinc20.docking.org/catalogs/{catalog}/substances/?page={x}')
  
  int.id <- read_html(url) |>
    html_elements('#print a') |>
    html_text2() |>
    str_extract('ZINC\\d+') |>
    str_extract('\\d+') |>
    str_pad(12, side = 'left', pad = '0')
  
  str_glue('ZINC{int.id}')
}

mcee.11k <- 1:110 |>
  map(safely(rvest_zinc_catalog), .progress = T)

mcee.111 <- rvest_zinc_catalog(111)

mcee.11k |>
  list_transpose() |>
  pluck('error') |>
  every(is_null)

mcee.11k <- mcee.11k |>
  list_transpose() |>
  pluck('result')

mcee.11k.intid <- mcee.11k |>
  list_c() |>
  str_extract('\\d+') |>
  str_pad(12, side = 'left', pad = '0')

mcee.11k.zincid <- mcee.11k.intid |>
  map_chr(\(x)str_glue('ZINC{x}'))

zinc20.cata.mcee <- tibble(zinc_id = c(mcee.11k.zincid, mcee.111)) |>
  write_csv('zinc20.cata.mcee.zincid.csv')

safe_zinc_catalog <-
safely(\(x, catalog)rvest_zinc_catalog(x = x, catalog = catalog), quiet = FALSE)

tic()
mce.9k <- 1:13 |>
  map(safe_zinc_catalog, .progress = T)
toc()

missed.index <- mce.9k |> list_transpose() |>
  pluck('error') |>
  set_names(1:13) |>
  discard(is_null) |>
  names() |>
  as.integer()

tic()
mce.m1 <- missed.index |>
  map(safe_zinc_catalog, .progress = T)
toc()

tic()
mce.m3 <- c(8,10) |>
  map(\(x)safe_zinc_catalog(x, 'mce'), .progress = T)
toc()

tic()
mce.1496 <- 14:96 |>
  map(safely(\(x)rvest_zinc_catalog(x = x, catalog = 'mce')), .progress = T)
toc()

missed.index2 <- mce.1496 |> list_transpose() |>
  pluck('error') |>
  set_names(14:96) |>
  discard(is_null) |>
  names() |>
  as.integer()

tic()
mce.m2 <- missed.index2 |>
  map(safely(\(x)rvest_zinc_catalog(x = x, catalog = 'mce')), .progress = T)
toc()

zinc20.cata.mce <- list(mce.9k, mce.1496, mce.m1, mce.m2, mce.m3) |>
  map(\(x)list_transpose(x) |> pluck('result')) |>
  list_c() |> list_c() |>
  tibble(zinc_id = _) |>
  write_csv('zinc20.cata.mce.zincid.csv')

zinc20.cata.mcee <- zinc20.cata.mcee |>
  mutate(src = 'mcee') 

zinc20.cata.mce |>
  mutate(src = 'mce') |>
  full_join(zinc20.cata.mcee, join_by(zinc_id)) |>
  unite(src, src.x:src.y, na.rm = T) |>
  write_csv('zinc20.cata.mce_mcee.zincid.csv')

zinc20.cata.mc3e <- read_tsv('zinc20.cata.mce_mcee.zincid.tsv')

mceee.inchi <- read_tsv('zinc20.mce_mcee.inchi.tsv')

## parse json in table --------
library(jsonlite)

parse_inchi_json <- function(x){
  x |>
    str_replace_all("\\'", '\\"') |>
    parse_json() |>
    pluck('inchi')
}

mceee.parsed <- mceee.inchi |>
  rowwise() |>
  mutate(InChI = parse_inchi_json(InChI))

mceee.parsed |> write_tsv('zinc20.mce_mcee.inchi.tsv')

mceee.parsed <- read_tsv('zinc20.mce_mcee.inchi.tsv')

mceee.parsed$InChI[685]
zinc20.cata.mc3e |> filter(!(zinc_id %in% mceee.parsed$zinc_id)) |>
  mutate(url = str_glue('https://zinc20.docking.org/substances/{zinc_id}.smi')) |>
  select(url) |>
  write_csv('mc3e.no.uci.smi.txt')

mc3e.no.uci <-
list.files('~/append-ssd/ringtail-test/data/mc3e-no-uci-smi', 'smi$',
           full.names = T) |>
  read_delim(col_names = c('smiles', 'zinc_id'))

mceee.parsed |> select(InChI) |>
  write_csv('zinc20.mce_mcee.inchi', col_names = F)

selleck <-
list.files('~/append-ssd/ringtail-test/data/H19/', 'selleck',
           full.names = T) |>
  read_csv() |>
  mutate(src = 'selleck')

cata.path <-
list.files('~/append-ssd/ringtail-test/data/H19/', 'zinc20.csv',
           full.names = T) |>
  str_subset('selleck|mcep2', negate = T)

cata.lst <- cata.path |>
  basename() |>
  str_remove('.zinc.+') |>
  set_names(cata.path, nm = _) |>
  imap(\(x, idx)read_csv(x) |> mutate(src = idx))

merge_catalogs <- function(df1, df2){
  message(str_glue('Merging {nrow(df1)} and {nrow(df2)} entries...'))
  
  df <- full_join(df1, df2, join_by(ZINC_ID)) |>
    unite(src, c(src.x, src.y), na.rm = T)
  
  message(str_glue('Find {nrow(df1)+nrow(df2)-nrow(df)} duplicates.'))
  message(str_glue('Keep {nrow(df)} entries.'))
  
  df
}

mc3e.slck <-
zinc20.cata.mc3e |>
  mutate(ZINC_ID = zinc_id, .keep = 'unused') |>
  merge_catalogs(selleck)

dedup.catalog <-
cata.lst |>
  reduce(merge_catalogs)

dedup.catalog <- dedup.catalog |>
  merge_catalogs(mc3e.slck)

od.catalog <- dedup.catalog |>
  mutate(on.demand = str_remove_all(src, '[a-z]+-v')) |>
  filter(!str_detect(on.demand, '[a-z]')) |>
  select(-on.demand) |>
  mutate(zinc_id = ZINC_ID, .keep = 'unused') |>
  write_csv('virtual_screen_v3/vendor4.on-demand.zincid.csv')

is.catalog <- dedup.catalog |>
  mutate(on.demand = str_remove_all(src, '[a-z]+-v')) |>
  filter(str_detect(on.demand, '[a-z]')) |>
  select(-on.demand) |>
  mutate(zinc_id = ZINC_ID, .keep = 'unused') |>
  write_csv('virtual_screen_v3/vendor4.in-stock.zincid.csv')

is.catalog <- read_csv('virtual_screen_v3/vendor4.in-stock.zincid.csv')

tm.metab <-
  read_delim('targetmolmetab-substances.smi', col_names = c('smi', 'zinc_id'))

is.append2 <- tm.metab |>
  filter(!(zinc_id %in% is.catalog$zinc_id))

is.append2 |>
  write_delim('instock.append2.smi', col_names = F)

is.inchi <- read_tsv('virtual_screen_v3/vendor4.in-stock.inchi.inchi.tsv')

is.catalog |>
  filter(!(zinc_id %in% is.inchi$zinc_id)) |>
  write_csv('vendor4.in-stock.nouci.id.csv')

od.inchi <- read_tsv('virtual_screen_v3/vendor4.on-demand.inchi.inchi.tsv')

od.catalog |>
  filter(!(zinc_id %in% od.inchi$zinc_id)) |>
  write_csv('vendor4.on-demand.nouci.id.csv')

is.todl <- read_csv('vendor4.in-stock.nouci.id.csv')
is.dled <- read_delim('vendor4.is.nouci.smi', col_names = F)

is.todl |> filter(!(zinc_id %in% is.dled$X2)) |>
  write_csv('is.todl2.csv')

pped.pdbqt <-
read_csv('~/append-ssd/ringtail-test/data/zinc20-vendor4-in-stock/prepared.lst',
         col_names = 'path')

pped.id <- pped.pdbqt |>
  mutate(zinc_id = str_extract(path, 'ZINC\\d+')) |>
  distinct(zinc_id)

scrubable.instock <-
read_delim('~/append-ssd/ringtail-test/data/zinc20-vendor4-in-stock/scrubed.smi',
           col_names = c('smi','zinc_id'))

scrubable.instock <- scrubable.instock |>
  mutate(zinc_id = str_extract(zinc_id, 'ZINC\\d+')) |>
  distinct(zinc_id)

scrubable.instock |>
  filter(!(zinc_id %in% pped.id$zinc_id)) |>
  left_join(all.instock) |>
  relocate(smi) |>
  write_delim('err.instock.smi', col_names = F)
